# -*- coding: utf-8 -*-
'''
Created on 27-11-2012

@author: LONG HOANG GIANG
'''
'''
-- Danh sách truyện: 
    + Mới nhất:
        http://mobileapi3.hixx.info/ws/home/danh_sach/page/1
    + A-Z:
        http://mobileapi4.hixx.info/ws/home/danh_sach/page/1/order_by/ASC
    + Search: (search: cmag)
        http://mobileapi4.hixx.info/ws/home/danh_sach/page/1/order_by/ASC/q/cmag
        http://mobileapi2.hixx.info/ws/home/danh_sach/page/1/q/cmag
- Danh sách chương:
    http://mobileapi4.hixx.info/ws/home/danh_sach_chuong/39763          -> id = 39763
    http://mobileapi3.hixx.info/ws/home/danh_sach_chuong/39763/page/1/order_by/ASC

- Chi tiết chương:
    http://mobileapi3.hixx.info/ws/home/chi_tiet_chuong/39764           -> id = 39764
'''
import sys
import os
sys.path.append(os.path.join(os.path.dirname(__file__), '../'))
from CrawlerLib import commonlib
from CrawlerLib.sqlitehelper import SQLiteHelper
import datetime
import simplejson as json

def getAllStories():
    urlfm = 'http://mobileapi3.hixx.info/ws/home/danh_sach/page/{0}'
    tree = commonlib.loadweb('http://mobileapi3.hixx.info/ws/home/danh_sach/page/1').build_tree(scheme='xml')
    maxPage = int(float(tree.xpath("//total_page/text()")[0]))
    datastoragefile = "/longhoanggiang/hixx/stories"
    try:
        fp = open(datastoragefile, 'r')
        data = json.load(fp)
        fp.close()
    except:
        data = None
    if data == None:
        data = []
        for page in range(1, maxPage + 1):
            url = urlfm.format(page)
            tree = commonlib.loadweb(url).build_tree(scheme='xml')
            for item in tree.xpath("//item"):
                id = item.xpath("./id/text()")[0]
                title = commonlib.stringify(item.xpath("./title")[0])
                isNew = int(float(item.xpath("./truyen_moi/text()")[0]))
                data.append({'id': id, 'title': title, 'isNew': isNew})
        f = open(datastoragefile, 'w')
        json.dump(data, f)
        f.close()
    for item in data:
        print item['id'], item['title'], item['isNew']
    return data
    
def getChapters(storyId):
    urlfm = 'http://mobileapi3.hixx.info/ws/home/danh_sach_chuong/{0}/page/{1}/order_by/ASC'
    cPage = 1
    data = []
    while cPage < 100:
        url = urlfm.format(storyId, cPage)
        tree = commonlib.loadweb(url).build_tree(scheme='xml')
        print commonlib.Etree.tostring(tree)
        for item in tree.xpath('//item'):
            title = commonlib.stringify(item.xpath('./title'))
            id = commonlib.stringify(item.xpath('./id'))
            data.append({'id': id, 'title': title})
            print id, title
        totalPage = int(float(tree.xpath("//total_page/text()")[0]))
        if totalPage <= cPage: break
        cPage += 1
    return data

def process():
    db = SQLiteHelper("databasehixx")
    db.execute("CREATE TABLE IF NOT EXISTS udv_stories(id INTEGER PRIMARY KEY, title VARCHAR(200), isNew INTEGER DEFAULT 0, kdTitle VARCHAR(200));")
    db.execute("CREATE INDEX IF NOT EXISTS idx_story_name_kd ON udv_stories(kdTitle)")
    db.execute("CREATE TABLE IF NOT EXISTS udv_chapters(id INTEGER PRIMARY KEY, sId INTEGER, name VARCHAR(255), filecontent TEXT)")
    db.execute("CREATE INDEX IF NOT EXISTS idx_chapter_story ON udv_chapters(sId)")
    
    
    
    db.commit()
    db.close()
    



if __name__ == '__main__':

#    getAllStories()
    getChapters('4736')
#    process()
    
    print '> Finished {0}'.format(datetime.datetime.now())
    os._exit(1)

